package com.searchdemo.webmagic;

import java.io.FileWriter;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.net.URL;
import java.net.URLDecoder;
import java.net.URLEncoder;
import java.util.Iterator;

import org.apache.commons.io.IOUtils;
import org.apache.http.HttpEntity;
import org.apache.http.HttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClientBuilder;
import org.apache.http.util.EntityUtils;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

public class ProductCrawlDemo {

	public static void main2(String[] args) throws IOException {
		URL url = new URL(
				"https://search.jd.com/Search?keyword=%E5%8F%A3%E7%BD%A9&enc=utf-8&qrst=1&rt=1&stop=1&vt=2&click=0");// 综合排序
		// https://search.jd.com/Search?keyword=%E5%8F%A3%E7%BD%A9&enc=utf-8&qrst=1&rt=1&stop=1&vt=2&offset=2&page=3&s=59&click=0
		// https://search.jd.com/Search?keyword=%E5%8F%A3%E7%BD%A9&enc=utf-8&qrst=1&rt=1&stop=1&vt=2&offset=2&psort=3&click=0
		// //销量
		// https://search.jd.com/Search?keyword=%E5%8F%A3%E7%BD%A9&enc=utf-8&qrst=1&rt=1&stop=1&vt=2&psort=4&click=0
		// //评论数
		// https://search.jd.com/Search?keyword=%E5%8F%A3%E7%BD%A9&enc=utf-8&qrst=1&rt=1&stop=1&vt=2&psort=5&click=0
		// //新品
		// https://search.jd.com/Search?keyword=%E5%8F%A3%E7%BD%A9&enc=utf-8&qrst=1&rt=1&stop=1&vt=2&psort=1&click=0
		// //价格 降序
		// https://search.jd.com/Search?keyword=%E5%8F%A3%E7%BD%A9&enc=utf-8&qrst=1&rt=1&stop=1&vt=2&psort=2&click=0
		// //价格 升序
		Elements link = Jsoup.parse(url, 200000).select(".gl-item");
		Iterator<Element> it = link.iterator();
		while (it.hasNext()) {
			System.out.println(it.next().html());
		}
	}

	public static void main3(String[] args) throws IOException {
		URL url = new URL(
				"https://search.jd.com/Search?keyword=%E5%8F%A3%E7%BD%A9&enc=utf-8&qrst=1&rt=1&stop=1&vt=2&offset=2&page=1&s=1&click=0");// 综合排序
		Elements link = Jsoup.parse(url, 2000).select(".gl-item");
		Iterator<Element> it = link.iterator();
		while (it.hasNext()) {
			Element ele = it.next();
			// System.out.println(ele.html());
			System.out.println(ele.attr("data-sku") + "\t"
					+ ele.select(".p-name>a>em").html() + "\t"
					+ ele.select(".p-price>strong>i").text());
			Elements detail_eles = ele.select(".p-img>a");
			System.out.println(detail_eles.attr("href"));

			String picurl = "";
			Elements img_eles = detail_eles.select("img");
			if (img_eles != null) {
				picurl = img_eles.attr("data-lazy-img");
				if (StringUtils.isEmpty(picurl)) {
					picurl = img_eles.attr("src");
				}
			}
			System.out.println(picurl);
		}
	}

	private static void parseHtml(String html) throws IOException {
		IOUtils.write(html, new FileWriter("D://a.html"));
		Elements link = Jsoup.parse(html).select(".gl-item");
		Iterator<Element> it = link.iterator();
		int i = 0;
		while (it.hasNext()) {
			Element ele = it.next();
			// System.out.println(ele.html());
			System.out.println(ele.attr("data-sku") + "\t"
					+ ele.select(".p-name>a>em").html() + "\t"
					+ ele.select(".p-price>strong>i").text());
			Elements detail_eles = ele.select(".p-img>a");
			System.out.println(detail_eles.attr("href"));

			String picurl = "";
			Elements img_eles = detail_eles.select("img");
			if (img_eles != null) {
				picurl = img_eles.attr("data-lazy-img");
				if (StringUtils.isEmpty(picurl)) {
					picurl = img_eles.attr("src");
				}
			}
			i++;
			System.out.println(picurl);
		}
		System.err.println(i);
	}

	public static void main(String[] args) throws IOException {
		String content = "";
		CloseableHttpClient httpclient = HttpClientBuilder.create().build();
		try {
			HttpGet httpget = new HttpGet(
					"https://search.jd.com/Search?keyword=%E5%8F%A3%E7%BD%A9&enc=utf-8&qrst=1&rt=1&stop=1&vt=2&offset=2&page=1&s=1&click=0");
			HttpResponse response = httpclient.execute(httpget);
			HttpEntity entity = response.getEntity();
			if (entity != null) {
				// System.out.println("Response content length: " +
				// entity.getContentLength());
				content = EntityUtils.toString(entity, "utf-8");
				// System.out.println("Response content: " + content);
			}
		} catch (Exception e) {
			e.printStackTrace();
		} finally {
			httpclient.close();
			parseHtml(content);
		}
	}

	public static void test() throws UnsupportedEncodingException {
		String url1 = "口罩";
		System.out.println(URLEncoder.encode(url1, "UTF-8"));

		String url = "%E5%8F%A3%E7%BD%A9";
		String urlStr = URLDecoder.decode(url, "UTF-8");
		System.out.println(urlStr);
	}

}
